/**
 * @author Fang Lu, fanglupku@gmail.com
 * 2011-3-25
 */
package code.extract.training;

import java.io.BufferedWriter;
import java.io.File;
import java.io.FileWriter;
import java.net.URLEncoder;
import java.util.ArrayList;
import java.util.Random;

import code.extract.CEConfig;
import code.extract.FileDealer;

import net.GooglePageParserRegexp;
import net.GoogleResultItem;
import net.PageFetcher;

public class TrainLinkCrawler {
	public static final String KEYWORD_START_SYMBOL = "#";
	public static void main(String args[]){
		/**
		 * 1.init page num and wait time
		 */
		int rnInPage = 100; 
		int GooglePages = CEConfig.getGoogleResultCount() / rnInPage;
		if (CEConfig.getGoogleResultCount() % rnInPage != 0){
			GooglePages += 1;
		}
		int maxWaitTime = CEConfig.getGoogleWaitSecondMax();
		int minWaitTime = CEConfig.getGoogleWaitSecondMin();
		Random timeRandom = new Random(System.currentTimeMillis());
		int timeDiff = maxWaitTime - minWaitTime;
		/**
		 * 2. init keyword list
		 */
		String googlePageSavePath = CEConfig.getGoogleResultSavepath();
		File googlePageSaveDir = new File(googlePageSavePath);
		if (!googlePageSaveDir.exists()){
			googlePageSaveDir.mkdirs();
		}
		String keywordListFilename = CEConfig.getGoogleKeywordListFilename();
		String keywordCrawledListFilename = CEConfig.getGoogleKeywordCrawledListFilename();
		KeywordList keywordList = new KeywordList(keywordListFilename, keywordCrawledListFilename, null);
		String googleResultFilename = CEConfig.getGoogleLinklistFilename();
		File linkFile = new File(googleResultFilename);
		File crawledListFile = new File(keywordCrawledListFilename);
		try{
			FileWriter link_fw = new FileWriter(linkFile, true);
			BufferedWriter link_bw = new BufferedWriter(link_fw);
			FileWriter crawled_fw = new FileWriter(crawledListFile, true);
			BufferedWriter crawled_bw = new BufferedWriter(crawled_fw);
//			System.out.println("keywordlist filename:" + keywordListFilename);
			System.out.println("size:" + keywordList.getContent().size());
			ArrayList<String> wordList = keywordList.getUncrawledContent();
			for (int i=0; i<wordList.size(); ++i){
				/**
				 * 3. construct the query
				 */
				String query = "\"" + wordList.get(i) + "\"+example+OR+java";
				query = URLEncoder.encode(query, "utf8");
				System.out.println("query:" + query);
				link_bw.write(KEYWORD_START_SYMBOL);
				link_bw.write(wordList.get(i));
				link_bw.write("\n");
				for (int j=0; j<GooglePages; ++j){
					String queryURL = "http://www.google.com/search?num=" + rnInPage + "&q="
										+ query + "&hl=en&start="
										+ (j * rnInPage) + "&sa=N";
					/**
					 * 4. fetch the google result
					 */
					String content = PageFetcher.fetchURL(queryURL);
					String filename = googlePageSavePath + File.separatorChar + i + "_" + j + ".google.html";
					try{
						FileDealer.saveFile(content, filename);
					}catch (Exception e) {
						// TODO: handle exception
						e.printStackTrace();
					}
					ArrayList<GoogleResultItem> items = new GooglePageParserRegexp().parse(content, "pp_" + j);
					for (GoogleResultItem googleResultItem : items) {
						link_bw.write(googleResultItem.getOriginalURL());
						link_bw.write("\n");
						
					}
					int thisWaitSecond = maxWaitTime;
					int thisRandomTime = timeRandom.nextInt() >>> 1;
					if (timeDiff <= 0){
						thisWaitSecond = minWaitTime;
					}else{
						
						thisWaitSecond = thisRandomTime % timeDiff + minWaitTime;
					}
//					System.out.println("sleep time: " + thisWaitSecond + ", min: " + minWaitTime + ", random: " + thisRandomTime + ", diff: " + timeDiff);
					Thread.sleep(thisWaitSecond * 1000);
				}
				link_bw.flush();
				crawled_bw.write(wordList.get(i));
				crawled_bw.write("\n");
				crawled_bw.flush();
			}
			link_bw.close();
			link_fw.close();
			crawled_bw.close();
			crawled_fw.close();
		}catch (Exception e) {
			// TODO: handle exception
			e.printStackTrace();
		}
	}

}
